package wordcount

import org.apache.spark.{SparkConf, SparkContext}

object Spark_WordCount_2 {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setMaster("local").setAppName("WordCount")
    val sc = new SparkContext(sparkConf);

//    var lines: RDD[String] = sc.textFile("datas")
//    val word: RDD[String] = lines.flatMap(_.split(" "))
//    val wordToOne = word.map {
//      word => (word, 1)
//    }
//    val wordGroup: RDD[(String, Iterable[(String, Int)])] = wordToOne.groupBy(t => t._1)
//    val wordToCount = wordGroup.map {
//      case (word, list) => {
//        list.reduce(
//          (t1, t2) => {
//            (t1._1, t1._2 + t2._2)
//          }
//        )
//      }
//    }
//    val array: Array[(String, Int)] = wordToCount.collect()
//    array.foreach(println)
    //读取文件
    val rdd1 = sc.textFile("datas")
    //文件拆分
    val rdd2 = rdd1.flatMap(item=>item.split(" "))
    //赋词频
    val rdd3 = rdd2.map(item=>(item,1))
    //以key为区分，词频相加
    val rdd4 = rdd3.reduceByKey((curr,agg)=>curr+agg)
    println(rdd4.toDebugString)
    //收集数据并且输出
//    rdd4.collect().foreach(println)
//    sc.stop()
  }
}
